# current restriction.
#2011 adhoc ONLY.
from operator import itemgetter, attrgetter

print "condition:2011,adhoc"
outputFeatureFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/query_features_2011_adhoc.txt"

outputPriorityListFileName = "/data1/team/weijiang/machine-learning-project-related/trec-related/trec2011/priorityWarcDocumentsIDs_2011_sorted.txt"


queryContentNumberDict = {}
inputFileNameForQueryContent = "/data1/team/weijiang/machine-learning-project-related/trec-related/trec2011/queries.101-150.txt"
inputFileHandlerForQueryContent = open(inputFileNameForQueryContent, "r")
for currentLine in inputFileHandlerForQueryContent:
    currentLineElements = currentLine.strip().split(":")
    queryNumber = int(currentLineElements[0])
    queryContent = currentLineElements[1]
    if queryNumber not in queryContentNumberDict:
        queryContentNumberDict[queryNumber] = queryContent



fileList = []

'''
#info at position 1
#2009
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/prels.1-50",1,2009) )
fileList.append(  ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/prels.catA.1-50",1,2009) )
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/prels.catB.1-50",1,2009) )


#info at position 2
#2009
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2009/qrels.diversity",2,2009))

#2010
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2010/10.adhoc-qrels.final",2,2010))
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2010/10.diversity-qrels.final",2,2010))
'''

#2011
fileList.append( ("/data1/team/weijiang/machine-learning-project-related/trec-related/trec2011/qrels.adhoc",2,2011))

warcDocumentIDFreqsDict = {}

outputQueryFeatureFileHandler = open(outputFeatureFileName,"w")

outputLineListForIDFreqs = []

for fileNameTuple in fileList:
    (fileName,position,year) = fileNameTuple
    print "processing ",fileName
    inputFileHandler1 = open(fileName,"r")
    for currentLine in inputFileHandler1.readlines():
        
        currentLineElements = currentLine.strip().split(" ")
        queryNumber = int(currentLineElements[0])
        diversitySubTopicNumber = int(currentLineElements[1])
        WARCTrecID = currentLineElements[2]
        relevenceScore = int(currentLineElements[3])
        outputLineInTupleFormat = (queryNumber, queryContentNumberDict[queryNumber], diversitySubTopicNumber, WARCTrecID, relevenceScore) 
        outputLineListForIDFreqs.append(outputLineInTupleFormat)
        
        if WARCTrecID not in warcDocumentIDFreqsDict:
            warcDocumentIDFreqsDict[WARCTrecID] = 0
        else:
            warcDocumentIDFreqsDict[WARCTrecID] += 1
    
    inputFileHandler1.close()

outputLineListForIDFreqs.sort(cmp=None, key=itemgetter(3), reverse=False)
for eachTuple in outputLineListForIDFreqs:
    (queryNumber, queryContent, diversitySubTopicNumber, WARCTrecID, relevenceScore) = eachTuple
    outputQueryFeatureFileHandler.write(str(queryNumber) + '\t' + queryContent + '\t' + str(diversitySubTopicNumber) + "\t" + WARCTrecID + "\t" + str(relevenceScore) + "\n")

outputQueryFeatureFileHandler.close()



maxValue = -1
mostFreqWarcDocumentID = ""

outputPriorityListFileHandler = open(outputPriorityListFileName,"w")
outputLineListForIDFreqs = []

for currentID in warcDocumentIDFreqsDict:
    outputLineListForIDFreqs.append( (currentID, warcDocumentIDFreqsDict[currentID]) )

outputLineListForIDFreqs.sort(cmp=None, key=itemgetter(0), reverse=False)
for currentElement in outputLineListForIDFreqs:
    (warcTrecID,freq) = currentElement
    outputLine = warcTrecID + " " + str(freq)
    outputPriorityListFileHandler.write(outputLine + "\n")
    if maxValue < freq:
        mostFreqWarcDocumentID, maxValue = warcTrecID,freq

outputPriorityListFileHandler.close()

print "General Info"
print "IDs Size:",len(warcDocumentIDFreqsDict)
print "mostFreqWarcDocumentID:",mostFreqWarcDocumentID
print "maxValue:",maxValue




    
    
    
    
    
    
    
